{
 "nbformat": 4,
 "nbformat_minor": 0,
 "metadata": {
  "colab": {
   "provenance": []
  },
  "kernelspec": {
   "name": "python3",
   "display_name": "Python 3"
  },
  "language_info": {
   "name": "python"
  }
 },
 "cells": [
  {
   "cell_type": "markdown",
   "source": [
    "# Ingesting data with OneLake Part I\n",
    "\n",
    "\n",
    "In this notebook we will learn how to index data from Microsoft OneLake [OneLake](https://learn.microsoft.com/en-us/fabric/onelake/onelake-overview) to Elasticsearch. This demonstration is related to the article [Ingesting data with OneLake Part I.](https://www.elastic.co/search-labs/blog/ingesting-data-with-onelake-part-i)\n"
   ],
   "metadata": {
    "id": "GHYboK3ekvbr"
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "Dzm-AXfcJNIT",
    "outputId": "7bb571fa-eb57-4b7e-c729-0bd95fb0a42a"
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Collecting elasticsearch==8.14\n",
      "  Downloading elasticsearch-8.14.0-py3-none-any.whl.metadata (7.2 kB)\n",
      "Collecting azure-identity\n",
      "  Downloading azure_identity-1.19.0-py3-none-any.whl.metadata (80 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m80.6/80.6 kB\u001b[0m \u001b[31m3.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting azure-storage-file-datalake\n",
      "  Downloading azure_storage_file_datalake-12.17.0-py3-none-any.whl.metadata (16 kB)\n",
      "Collecting azure-cli\n",
      "  Downloading azure_cli-2.65.0-py3-none-any.whl.metadata (8.3 kB)\n",
      "Collecting python-docx\n",
      "  Downloading python_docx-1.1.2-py3-none-any.whl.metadata (2.0 kB)\n",
      "Collecting elastic-transport<9,>=8.13 (from elasticsearch==8.14)\n",
      "  Downloading elastic_transport-8.15.1-py3-none-any.whl.metadata (3.7 kB)\n",
      "Collecting azure-core>=1.31.0 (from azure-identity)\n",
      "  Downloading azure_core-1.32.0-py3-none-any.whl.metadata (39 kB)\n",
      "Requirement already satisfied: cryptography>=2.5 in /usr/local/lib/python3.10/dist-packages (from azure-identity) (43.0.3)\n",
      "Collecting msal>=1.30.0 (from azure-identity)\n",
      "  Downloading msal-1.31.0-py3-none-any.whl.metadata (11 kB)\n",
      "Collecting msal-extensions>=1.2.0 (from azure-identity)\n",
      "  Downloading msal_extensions-1.2.0-py3-none-any.whl.metadata (7.6 kB)\n",
      "Requirement already satisfied: typing-extensions>=4.0.0 in /usr/local/lib/python3.10/dist-packages (from azure-identity) (4.12.2)\n",
      "Collecting azure-storage-blob>=12.23.0 (from azure-storage-file-datalake)\n",
      "  Downloading azure_storage_blob-12.23.1-py3-none-any.whl.metadata (26 kB)\n",
      "Collecting isodate>=0.6.1 (from azure-storage-file-datalake)\n",
      "  Downloading isodate-0.7.2-py3-none-any.whl.metadata (11 kB)\n",
      "Collecting antlr4-python3-runtime~=4.13.1 (from azure-cli)\n",
      "  Downloading antlr4_python3_runtime-4.13.2-py3-none-any.whl.metadata (304 bytes)\n",
      "Collecting azure-appconfiguration~=1.7.0 (from azure-cli)\n",
      "  Downloading azure_appconfiguration-1.7.1-py3-none-any.whl.metadata (27 kB)\n",
      "Collecting azure-batch~=14.2.0 (from azure-cli)\n",
      "  Downloading azure_batch-14.2.0-py3-none-any.whl.metadata (22 kB)\n",
      "Collecting azure-cli-core==2.65.0 (from azure-cli)\n",
      "  Downloading azure_cli_core-2.65.0-py3-none-any.whl.metadata (1.8 kB)\n",
      "Collecting azure-cosmos>=3.0.2,~=3.0 (from azure-cli)\n",
      "  Downloading azure_cosmos-3.2.0-py2.py3-none-any.whl.metadata (22 kB)\n",
      "Collecting azure-data-tables==12.4.0 (from azure-cli)\n",
      "  Downloading azure_data_tables-12.4.0-py3-none-any.whl.metadata (31 kB)\n",
      "Collecting azure-datalake-store~=0.0.53 (from azure-cli)\n",
      "  Downloading azure_datalake_store-0.0.53-py2.py3-none-any.whl.metadata (19 kB)\n",
      "Collecting azure-graphrbac~=0.60.0 (from azure-cli)\n",
      "  Downloading azure_graphrbac-0.60.0-py2.py3-none-any.whl.metadata (10 kB)\n",
      "Collecting azure-keyvault-administration==4.4.0b2 (from azure-cli)\n",
      "  Downloading azure_keyvault_administration-4.4.0b2-py3-none-any.whl.metadata (30 kB)\n",
      "Collecting azure-keyvault-certificates==4.7.0 (from azure-cli)\n",
      "  Downloading azure_keyvault_certificates-4.7.0-py3-none-any.whl.metadata (34 kB)\n",
      "Collecting azure-keyvault-keys==4.9.0b3 (from azure-cli)\n",
      "  Downloading azure_keyvault_keys-4.9.0b3-py3-none-any.whl.metadata (46 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m46.5/46.5 kB\u001b[0m \u001b[31m2.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting azure-keyvault-secrets==4.7.0 (from azure-cli)\n",
      "  Downloading azure_keyvault_secrets-4.7.0-py3-none-any.whl.metadata (28 kB)\n",
      "Collecting azure-mgmt-advisor==9.0.0 (from azure-cli)\n",
      "  Downloading azure_mgmt_advisor-9.0.0-py2.py3-none-any.whl.metadata (8.6 kB)\n",
      "Collecting azure-mgmt-apimanagement==4.0.0 (from azure-cli)\n",
      "  Downloading azure_mgmt_apimanagement-4.0.0-py3-none-any.whl.metadata (24 kB)\n",
      "Collecting azure-mgmt-appconfiguration==3.0.0 (from azure-cli)\n",
      "  Downloading azure_mgmt_appconfiguration-3.0.0-py3-none-any.whl.metadata (9.3 kB)\n",
      "Collecting azure-mgmt-appcontainers==2.0.0 (from azure-cli)\n",
      "  Downloading azure_mgmt_appcontainers-2.0.0-py3-none-any.whl.metadata (19 kB)\n",
      "Collecting azure-mgmt-applicationinsights~=1.0.0 (from azure-cli)\n",
      "  Downloading azure_mgmt_applicationinsights-1.0.0-py2.py3-none-any.whl.metadata (11 kB)\n",
      "Collecting azure-mgmt-authorization~=4.0.0 (from azure-cli)\n",
      "  Downloading azure_mgmt_authorization-4.0.0-py3-none-any.whl.metadata (18 kB)\n",
      "Collecting azure-mgmt-batchai==7.0.0b1 (from azure-cli)\n",
      "  Downloading azure_mgmt_batchai-7.0.0b1-py2.py3-none-any.whl.metadata (8.9 kB)\n",
      "Collecting azure-mgmt-batch~=17.3.0 (from azure-cli)\n",
      "  Downloading azure_mgmt_batch-17.3.0-py3-none-any.whl.metadata (19 kB)\n",
      "Collecting azure-mgmt-billing==6.0.0 (from azure-cli)\n",
      "  Downloading azure_mgmt_billing-6.0.0-py2.py3-none-any.whl.metadata (7.5 kB)\n",
      "Collecting azure-mgmt-botservice~=2.0.0b3 (from azure-cli)\n",
      "  Downloading azure_mgmt_botservice-2.0.0-py3-none-any.whl.metadata (18 kB)\n",
      "Collecting azure-mgmt-cdn==12.0.0 (from azure-cli)\n",
      "  Downloading azure_mgmt_cdn-12.0.0-py3-none-any.whl.metadata (25 kB)\n",
      "Collecting azure-mgmt-cognitiveservices~=13.5.0 (from azure-cli)\n",
      "  Downloading azure_mgmt_cognitiveservices-13.5.0-py3-none-any.whl.metadata (18 kB)\n",
      "Collecting azure-mgmt-compute~=33.0.0 (from azure-cli)\n",
      "  Downloading azure_mgmt_compute-33.0.0-py3-none-any.whl.metadata (69 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m69.4/69.4 kB\u001b[0m \u001b[31m4.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting azure-mgmt-containerinstance==10.1.0 (from azure-cli)\n",
      "  Downloading azure_mgmt_containerinstance-10.1.0-py3-none-any.whl.metadata (13 kB)\n",
      "Collecting azure-mgmt-containerregistry==10.3.0 (from azure-cli)\n",
      "  Downloading azure_mgmt_containerregistry-10.3.0-py3-none-any.whl.metadata (23 kB)\n",
      "Collecting azure-mgmt-containerservice~=32.0.0 (from azure-cli)\n",
      "  Downloading azure_mgmt_containerservice-32.0.0-py3-none-any.whl.metadata (52 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m52.0/52.0 kB\u001b[0m \u001b[31m2.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting azure-mgmt-cosmosdb==9.6.0 (from azure-cli)\n",
      "  Downloading azure_mgmt_cosmosdb-9.6.0-py3-none-any.whl.metadata (108 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m108.2/108.2 kB\u001b[0m \u001b[31m6.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting azure-mgmt-databoxedge~=1.0.0 (from azure-cli)\n",
      "  Downloading azure_mgmt_databoxedge-1.0.0-py2.py3-none-any.whl.metadata (8.4 kB)\n",
      "Collecting azure-mgmt-datamigration~=10.0.0 (from azure-cli)\n",
      "  Downloading azure_mgmt_datamigration-10.0.0-py2.py3-none-any.whl.metadata (13 kB)\n",
      "Collecting azure-mgmt-devtestlabs~=4.0 (from azure-cli)\n",
      "  Downloading azure_mgmt_devtestlabs-4.0.0-py2.py3-none-any.whl.metadata (10 kB)\n",
      "Collecting azure-mgmt-dns~=8.0.0 (from azure-cli)\n",
      "  Downloading azure_mgmt_dns-8.0.0-py2.py3-none-any.whl.metadata (8.9 kB)\n",
      "Collecting azure-mgmt-eventgrid==10.2.0b2 (from azure-cli)\n",
      "  Downloading azure_mgmt_eventgrid-10.2.0b2-py3-none-any.whl.metadata (23 kB)\n",
      "Collecting azure-mgmt-eventhub~=10.1.0 (from azure-cli)\n",
      "  Downloading azure_mgmt_eventhub-10.1.0-py3-none-any.whl.metadata (15 kB)\n",
      "Collecting azure-mgmt-extendedlocation==1.0.0b2 (from azure-cli)\n",
      "  Downloading azure_mgmt_extendedlocation-1.0.0b2-py2.py3-none-any.whl.metadata (2.4 kB)\n",
      "Collecting azure-mgmt-hdinsight==9.0.0b3 (from azure-cli)\n",
      "  Downloading azure_mgmt_hdinsight-9.0.0b3-py3-none-any.whl.metadata (15 kB)\n",
      "Collecting azure-mgmt-imagebuilder~=1.3.0 (from azure-cli)\n",
      "  Downloading azure_mgmt_imagebuilder-1.3.0-py3-none-any.whl.metadata (9.4 kB)\n",
      "Collecting azure-mgmt-iotcentral~=10.0.0b1 (from azure-cli)\n",
      "  Downloading azure_mgmt_iotcentral-10.0.0b2-py3-none-any.whl.metadata (8.3 kB)\n",
      "Collecting azure-mgmt-iothub==3.0.0 (from azure-cli)\n",
      "  Downloading azure_mgmt_iothub-3.0.0-py3-none-any.whl.metadata (16 kB)\n",
      "Collecting azure-mgmt-iothubprovisioningservices==1.1.0 (from azure-cli)\n",
      "  Downloading azure_mgmt_iothubprovisioningservices-1.1.0-py3-none-any.whl.metadata (7.3 kB)\n",
      "Collecting azure-mgmt-keyvault==10.3.0 (from azure-cli)\n",
      "  Downloading azure_mgmt_keyvault-10.3.0-py3-none-any.whl.metadata (15 kB)\n",
      "Collecting azure-mgmt-kusto~=0.3.0 (from azure-cli)\n",
      "  Downloading azure_mgmt_kusto-0.3.0-py2.py3-none-any.whl.metadata (4.7 kB)\n",
      "Collecting azure-mgmt-loganalytics==13.0.0b4 (from azure-cli)\n",
      "  Downloading azure_mgmt_loganalytics-13.0.0b4-py3-none-any.whl.metadata (17 kB)\n",
      "Collecting azure-mgmt-managementgroups~=1.0.0 (from azure-cli)\n",
      "  Downloading azure_mgmt_managementgroups-1.0.0-py2.py3-none-any.whl.metadata (5.6 kB)\n",
      "Collecting azure-mgmt-maps~=2.0.0 (from azure-cli)\n",
      "  Downloading azure_mgmt_maps-2.0.0-py2.py3-none-any.whl.metadata (5.6 kB)\n",
      "Collecting azure-mgmt-marketplaceordering==1.1.0 (from azure-cli)\n",
      "  Downloading azure_mgmt_marketplaceordering-1.1.0-py2.py3-none-any.whl.metadata (7.5 kB)\n",
      "Collecting azure-mgmt-media~=9.0 (from azure-cli)\n",
      "  Downloading azure_mgmt_media-9.0.0-py3-none-any.whl.metadata (14 kB)\n",
      "Collecting azure-mgmt-monitor~=5.0.0 (from azure-cli)\n",
      "  Downloading azure_mgmt_monitor-5.0.1-py3-none-any.whl.metadata (24 kB)\n",
      "Collecting azure-mgmt-msi~=7.0.0 (from azure-cli)\n",
      "  Downloading azure_mgmt_msi-7.0.0-py3-none-any.whl.metadata (8.9 kB)\n",
      "Collecting azure-mgmt-netapp~=10.1.0 (from azure-cli)\n",
      "  Downloading azure_mgmt_netapp-10.1.0-py3-none-any.whl.metadata (21 kB)\n",
      "Collecting azure-mgmt-policyinsights==1.1.0b4 (from azure-cli)\n",
      "  Downloading azure_mgmt_policyinsights-1.1.0b4-py3-none-any.whl.metadata (11 kB)\n",
      "Collecting azure-mgmt-privatedns~=1.0.0 (from azure-cli)\n",
      "  Downloading azure_mgmt_privatedns-1.0.0-py2.py3-none-any.whl.metadata (4.9 kB)\n",
      "Collecting azure-mgmt-rdbms~=10.2.0b16 (from azure-cli)\n",
      "  Downloading azure_mgmt_rdbms-10.2.0b18-py3-none-any.whl.metadata (33 kB)\n",
      "Collecting azure-mgmt-recoveryservicesbackup~=9.1.0 (from azure-cli)\n",
      "  Downloading azure_mgmt_recoveryservicesbackup-9.1.0-py3-none-any.whl.metadata (36 kB)\n",
      "Collecting azure-mgmt-recoveryservices~=3.0.0 (from azure-cli)\n",
      "  Downloading azure_mgmt_recoveryservices-3.0.0-py3-none-any.whl.metadata (13 kB)\n",
      "Collecting azure-mgmt-redhatopenshift~=1.5.0 (from azure-cli)\n",
      "  Downloading azure_mgmt_redhatopenshift-1.5.0-py3-none-any.whl.metadata (7.0 kB)\n",
      "Collecting azure-mgmt-redis~=14.4.0 (from azure-cli)\n",
      "  Downloading azure_mgmt_redis-14.4.0-py3-none-any.whl.metadata (16 kB)\n",
      "Collecting azure-mgmt-resource==23.1.1 (from azure-cli)\n",
      "  Downloading azure_mgmt_resource-23.1.1-py3-none-any.whl.metadata (37 kB)\n",
      "Collecting azure-mgmt-search~=9.0 (from azure-cli)\n",
      "  Downloading azure_mgmt_search-9.1.0-py3-none-any.whl.metadata (10 kB)\n",
      "Collecting azure-mgmt-security==6.0.0 (from azure-cli)\n",
      "  Downloading azure_mgmt_security-6.0.0-py3-none-any.whl.metadata (31 kB)\n",
      "Collecting azure-mgmt-servicebus~=8.2.0 (from azure-cli)\n",
      "  Downloading azure_mgmt_servicebus-8.2.0-py3-none-any.whl.metadata (15 kB)\n",
      "Collecting azure-mgmt-servicefabricmanagedclusters==2.0.0b6 (from azure-cli)\n",
      "  Downloading azure_mgmt_servicefabricmanagedclusters-2.0.0b6-py3-none-any.whl.metadata (9.1 kB)\n",
      "Collecting azure-mgmt-servicelinker==1.2.0b2 (from azure-cli)\n",
      "  Downloading azure_mgmt_servicelinker-1.2.0b2-py3-none-any.whl.metadata (7.2 kB)\n",
      "Collecting azure-mgmt-servicefabric~=2.1.0 (from azure-cli)\n",
      "  Downloading azure_mgmt_servicefabric-2.1.0-py3-none-any.whl.metadata (13 kB)\n",
      "Collecting azure-mgmt-signalr==2.0.0b1 (from azure-cli)\n",
      "  Downloading azure_mgmt_signalr-2.0.0b1-py3-none-any.whl.metadata (9.9 kB)\n",
      "Collecting azure-mgmt-sqlvirtualmachine==1.0.0b5 (from azure-cli)\n",
      "  Downloading azure_mgmt_sqlvirtualmachine-1.0.0b5-py3-none-any.whl.metadata (8.8 kB)\n",
      "Collecting azure-mgmt-sql==4.0.0b19 (from azure-cli)\n",
      "  Downloading azure_mgmt_sql-4.0.0b19-py3-none-any.whl.metadata (70 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m70.7/70.7 kB\u001b[0m \u001b[31m5.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting azure-mgmt-storage==21.2.0 (from azure-cli)\n",
      "  Downloading azure_mgmt_storage-21.2.0-py3-none-any.whl.metadata (30 kB)\n",
      "Collecting azure-mgmt-synapse==2.1.0b5 (from azure-cli)\n",
      "  Downloading azure_mgmt_synapse-2.1.0b5-py3-none-any.whl.metadata (15 kB)\n",
      "Collecting azure-mgmt-trafficmanager~=1.0.0 (from azure-cli)\n",
      "  Downloading azure_mgmt_trafficmanager-1.0.0-py3-none-any.whl.metadata (7.8 kB)\n",
      "Collecting azure-mgmt-web==7.2.0 (from azure-cli)\n",
      "  Downloading azure_mgmt_web-7.2.0-py3-none-any.whl.metadata (87 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m87.3/87.3 kB\u001b[0m \u001b[31m6.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hCollecting azure-monitor-query==1.2.0 (from azure-cli)\n",
      "  Downloading azure_monitor_query-1.2.0-py3-none-any.whl.metadata (32 kB)\n",
      "Collecting azure-multiapi-storage~=1.3.0 (from azure-cli)\n",
      "  Downloading azure_multiapi_storage-1.3.0-py2.py3-none-any.whl.metadata (6.5 kB)\n",
      "Collecting azure-storage-common~=1.4 (from azure-cli)\n",
      "  Downloading azure_storage_common-1.4.2-py2.py3-none-any.whl.metadata (8.1 kB)\n",
      "Collecting azure-synapse-accesscontrol~=0.5.0 (from azure-cli)\n",
      "  Downloading azure_synapse_accesscontrol-0.5.0-py2.py3-none-any.whl.metadata (2.1 kB)\n",
      "Collecting azure-synapse-artifacts~=0.19.0 (from azure-cli)\n",
      "  Downloading azure_synapse_artifacts-0.19.0-py3-none-any.whl.metadata (17 kB)\n",
      "Collecting azure-synapse-managedprivateendpoints~=0.4.0 (from azure-cli)\n",
      "  Downloading azure_synapse_managedprivateendpoints-0.4.0-py2.py3-none-any.whl.metadata (2.3 kB)\n",
      "Collecting azure-synapse-spark~=0.2.0 (from azure-cli)\n",
      "  Downloading azure_synapse_spark-0.2.0-py2.py3-none-any.whl.metadata (1.8 kB)\n",
      "Requirement already satisfied: chardet~=5.2.0 in /usr/local/lib/python3.10/dist-packages (from azure-cli) (5.2.0)\n",
      "Collecting colorama~=0.4.4 (from azure-cli)\n",
      "  Downloading colorama-0.4.6-py2.py3-none-any.whl.metadata (17 kB)\n",
      "Collecting fabric~=3.2.2 (from azure-cli)\n",
      "  Downloading fabric-3.2.2-py3-none-any.whl.metadata (3.5 kB)\n",
      "Collecting javaproperties~=0.5.1 (from azure-cli)\n",
      "  Downloading javaproperties-0.5.2-py2.py3-none-any.whl.metadata (5.8 kB)\n",
      "Collecting jsondiff~=2.0.0 (from azure-cli)\n",
      "  Downloading jsondiff-2.0.0-py3-none-any.whl.metadata (562 bytes)\n",
      "Requirement already satisfied: packaging>=20.9 in /usr/local/lib/python3.10/dist-packages (from azure-cli) (24.1)\n",
      "Collecting pycomposefile>=0.0.29 (from azure-cli)\n",
      "  Downloading pycomposefile-0.0.32-py3-none-any.whl.metadata (356 bytes)\n",
      "Collecting PyGithub~=1.38 (from azure-cli)\n",
      "  Downloading PyGithub-1.59.1-py3-none-any.whl.metadata (2.0 kB)\n",
      "Collecting PyNaCl~=1.5.0 (from azure-cli)\n",
      "  Downloading PyNaCl-1.5.0-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl.metadata (8.6 kB)\n",
      "Collecting scp~=0.13.2 (from azure-cli)\n",
      "  Downloading scp-0.13.6-py2.py3-none-any.whl.metadata (4.5 kB)\n",
      "Collecting semver==2.13.0 (from azure-cli)\n",
      "  Downloading semver-2.13.0-py2.py3-none-any.whl.metadata (5.0 kB)\n",
      "Requirement already satisfied: setuptools in /usr/local/lib/python3.10/dist-packages (from azure-cli) (75.1.0)\n",
      "Requirement already satisfied: six>=1.10.0 in /usr/local/lib/python3.10/dist-packages (from azure-cli) (1.16.0)\n",
      "Collecting sshtunnel~=0.1.4 (from azure-cli)\n",
      "  Downloading sshtunnel-0.1.5-py2.py3-none-any.whl.metadata (17 kB)\n",
      "Requirement already satisfied: tabulate in /usr/local/lib/python3.10/dist-packages (from azure-cli) (0.9.0)\n",
      "Requirement already satisfied: urllib3 in /usr/local/lib/python3.10/dist-packages (from azure-cli) (2.2.3)\n",
      "Collecting websocket-client~=1.3.1 (from azure-cli)\n",
      "  Downloading websocket_client-1.3.3-py3-none-any.whl.metadata (6.9 kB)\n",
      "Collecting xmltodict~=0.12 (from azure-cli)\n",
      "  Downloading xmltodict-0.14.2-py2.py3-none-any.whl.metadata (8.0 kB)\n",
      "Requirement already satisfied: distro in /usr/local/lib/python3.10/dist-packages (from azure-cli) (1.9.0)\n",
      "Collecting argcomplete~=3.3.0 (from azure-cli-core==2.65.0->azure-cli)\n",
      "  Downloading argcomplete-3.3.0-py3-none-any.whl.metadata (16 kB)\n",
      "Collecting azure-cli-telemetry==1.1.0.* (from azure-cli-core==2.65.0->azure-cli)\n",
      "  Downloading azure_cli_telemetry-1.1.0-py3-none-any.whl.metadata (1.8 kB)\n",
      "Collecting azure-mgmt-core<2,>=1.2.0 (from azure-cli-core==2.65.0->azure-cli)\n",
      "  Downloading azure_mgmt_core-1.5.0-py3-none-any.whl.metadata (4.3 kB)\n",
      "Collecting humanfriendly~=10.0 (from azure-cli-core==2.65.0->azure-cli)\n",
      "  Downloading humanfriendly-10.0-py2.py3-none-any.whl.metadata (9.2 kB)\n",
      "Collecting jmespath (from azure-cli-core==2.65.0->azure-cli)\n",
      "  Downloading jmespath-1.0.1-py3-none-any.whl.metadata (7.6 kB)\n",
      "Collecting knack~=0.11.0 (from azure-cli-core==2.65.0->azure-cli)\n",
      "  Downloading knack-0.11.0-py3-none-any.whl.metadata (5.2 kB)\n",
      "Collecting msrestazure~=0.6.4 (from azure-cli-core==2.65.0->azure-cli)\n",
      "  Downloading msrestazure-0.6.4.post1-py2.py3-none-any.whl.metadata (15 kB)\n",
      "Collecting paramiko<4.0.0,>=2.0.8 (from azure-cli-core==2.65.0->azure-cli)\n",
      "  Downloading paramiko-3.5.0-py3-none-any.whl.metadata (4.4 kB)\n",
      "Collecting pkginfo>=1.5.0.1 (from azure-cli-core==2.65.0->azure-cli)\n",
      "  Downloading pkginfo-1.11.2-py3-none-any.whl.metadata (11 kB)\n",
      "Requirement already satisfied: PyJWT>=2.1.0 in /usr/local/lib/python3.10/dist-packages (from azure-cli-core==2.65.0->azure-cli) (2.9.0)\n",
      "Requirement already satisfied: pyopenssl>=17.1.0 in /usr/local/lib/python3.10/dist-packages (from azure-cli-core==2.65.0->azure-cli) (24.2.1)\n",
      "Requirement already satisfied: requests[socks] in /usr/local/lib/python3.10/dist-packages (from azure-cli-core==2.65.0->azure-cli) (2.32.3)\n",
      "Collecting microsoft-security-utilities-secret-masker~=1.0.0b2 (from azure-cli-core==2.65.0->azure-cli)\n",
      "  Downloading microsoft_security_utilities_secret_masker-1.0.0b3-py3-none-any.whl.metadata (3.2 kB)\n",
      "Requirement already satisfied: psutil>=5.9 in /usr/local/lib/python3.10/dist-packages (from azure-cli-core==2.65.0->azure-cli) (5.9.5)\n",
      "Collecting msrest>=0.6.21 (from azure-data-tables==12.4.0->azure-cli)\n",
      "  Downloading msrest-0.7.1-py3-none-any.whl.metadata (21 kB)\n",
      "Collecting azure-common~=1.1 (from azure-keyvault-administration==4.4.0b2->azure-cli)\n",
      "  Downloading azure_common-1.1.28-py2.py3-none-any.whl.metadata (5.0 kB)\n",
      "Collecting portalocker<3,>=1.4 (from msal-extensions>=1.2.0->azure-identity)\n",
      "  Downloading portalocker-2.10.1-py3-none-any.whl.metadata (8.5 kB)\n",
      "Collecting applicationinsights<0.12,>=0.11.1 (from azure-cli-telemetry==1.1.0.*->azure-cli-core==2.65.0->azure-cli)\n",
      "  Downloading applicationinsights-0.11.10-py2.py3-none-any.whl.metadata (982 bytes)\n",
      "Requirement already satisfied: lxml>=3.1.0 in /usr/local/lib/python3.10/dist-packages (from python-docx) (5.3.0)\n",
      "Requirement already satisfied: cffi in /usr/local/lib/python3.10/dist-packages (from azure-datalake-store~=0.0.53->azure-cli) (1.17.1)\n",
      "Requirement already satisfied: python-dateutil in /usr/local/lib/python3.10/dist-packages (from azure-multiapi-storage~=1.3.0->azure-cli) (2.8.2)\n",
      "Requirement already satisfied: certifi in /usr/local/lib/python3.10/dist-packages (from elastic-transport<9,>=8.13->elasticsearch==8.14) (2024.8.30)\n",
      "Collecting invoke>=2.0 (from fabric~=3.2.2->azure-cli)\n",
      "  Downloading invoke-2.2.0-py3-none-any.whl.metadata (3.3 kB)\n",
      "Collecting decorator>=5 (from fabric~=3.2.2->azure-cli)\n",
      "  Downloading decorator-5.1.1-py3-none-any.whl.metadata (4.0 kB)\n",
      "Requirement already satisfied: deprecated>=1.2 in /usr/local/lib/python3.10/dist-packages (from fabric~=3.2.2->azure-cli) (1.2.14)\n",
      "Requirement already satisfied: pyyaml in /usr/local/lib/python3.10/dist-packages (from pycomposefile>=0.0.29->azure-cli) (6.0.2)\n",
      "Requirement already satisfied: pycparser in /usr/local/lib/python3.10/dist-packages (from cffi->azure-datalake-store~=0.0.53->azure-cli) (2.22)\n",
      "Requirement already satisfied: wrapt<2,>=1.10 in /usr/local/lib/python3.10/dist-packages (from deprecated>=1.2->fabric~=3.2.2->azure-cli) (1.16.0)\n",
      "Requirement already satisfied: pygments in /usr/local/lib/python3.10/dist-packages (from knack~=0.11.0->azure-cli-core==2.65.0->azure-cli) (2.18.0)\n",
      "Requirement already satisfied: requests-oauthlib>=0.5.0 in /usr/local/lib/python3.10/dist-packages (from msrest>=0.6.21->azure-data-tables==12.4.0->azure-cli) (1.3.1)\n",
      "Collecting adal<2.0.0,>=0.6.0 (from msrestazure~=0.6.4->azure-cli-core==2.65.0->azure-cli)\n",
      "  Downloading adal-1.2.7-py2.py3-none-any.whl.metadata (6.9 kB)\n",
      "Collecting bcrypt>=3.2 (from paramiko<4.0.0,>=2.0.8->azure-cli-core==2.65.0->azure-cli)\n",
      "  Downloading bcrypt-4.2.0-cp39-abi3-manylinux_2_28_x86_64.whl.metadata (9.6 kB)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests[socks]->azure-cli-core==2.65.0->azure-cli) (3.4.0)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests[socks]->azure-cli-core==2.65.0->azure-cli) (3.10)\n",
      "Requirement already satisfied: PySocks!=1.5.7,>=1.5.6 in /usr/local/lib/python3.10/dist-packages (from requests[socks]->azure-cli-core==2.65.0->azure-cli) (1.7.1)\n",
      "Requirement already satisfied: oauthlib>=3.0.0 in /usr/local/lib/python3.10/dist-packages (from requests-oauthlib>=0.5.0->msrest>=0.6.21->azure-data-tables==12.4.0->azure-cli) (3.2.2)\n",
      "Downloading elasticsearch-8.14.0-py3-none-any.whl (480 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m480.2/480.2 kB\u001b[0m \u001b[31m26.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading azure_identity-1.19.0-py3-none-any.whl (187 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m187.6/187.6 kB\u001b[0m \u001b[31m13.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading azure_storage_file_datalake-12.17.0-py3-none-any.whl (255 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m255.7/255.7 kB\u001b[0m \u001b[31m20.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading azure_cli-2.65.0-py3-none-any.whl (14.3 MB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m14.3/14.3 MB\u001b[0m \u001b[31m60.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading azure_cli_core-2.65.0-py3-none-any.whl (257 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m257.7/257.7 kB\u001b[0m \u001b[31m15.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading azure_data_tables-12.4.0-py3-none-any.whl (113 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m113.9/113.9 kB\u001b[0m \u001b[31m7.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading azure_keyvault_administration-4.4.0b2-py3-none-any.whl (105 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m105.1/105.1 kB\u001b[0m \u001b[31m8.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading azure_keyvault_certificates-4.7.0-py3-none-any.whl (428 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m428.1/428.1 kB\u001b[0m \u001b[31m24.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading azure_keyvault_keys-4.9.0b3-py3-none-any.whl (149 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m149.5/149.5 kB\u001b[0m \u001b[31m9.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading azure_keyvault_secrets-4.7.0-py3-none-any.whl (348 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m348.6/348.6 kB\u001b[0m \u001b[31m20.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading azure_mgmt_advisor-9.0.0-py2.py3-none-any.whl (46 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m46.9/46.9 kB\u001b[0m \u001b[31m3.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading azure_mgmt_apimanagement-4.0.0-py3-none-any.whl (804 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m804.5/804.5 kB\u001b[0m \u001b[31m38.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading azure_mgmt_appconfiguration-3.0.0-py3-none-any.whl (252 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m252.0/252.0 kB\u001b[0m \u001b[31m17.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading azure_mgmt_appcontainers-2.0.0-py3-none-any.whl (214 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m214.1/214.1 kB\u001b[0m \u001b[31m13.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading azure_mgmt_batchai-7.0.0b1-py2.py3-none-any.whl (99 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m99.1/99.1 kB\u001b[0m \u001b[31m7.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading azure_mgmt_billing-6.0.0-py2.py3-none-any.whl (166 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m167.0/167.0 kB\u001b[0m \u001b[31m12.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading azure_mgmt_cdn-12.0.0-py3-none-any.whl (239 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m239.4/239.4 kB\u001b[0m \u001b[31m16.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading azure_mgmt_containerinstance-10.1.0-py3-none-any.whl (87 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m87.3/87.3 kB\u001b[0m \u001b[31m7.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading azure_mgmt_containerregistry-10.3.0-py3-none-any.whl (2.3 MB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.3/2.3 MB\u001b[0m \u001b[31m51.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading azure_mgmt_cosmosdb-9.6.0-py3-none-any.whl (387 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m387.9/387.9 kB\u001b[0m \u001b[31m18.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading azure_mgmt_eventgrid-10.2.0b2-py3-none-any.whl (248 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m248.5/248.5 kB\u001b[0m \u001b[31m12.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading azure_mgmt_extendedlocation-1.0.0b2-py2.py3-none-any.whl (37 kB)\n",
      "Downloading azure_mgmt_hdinsight-9.0.0b3-py3-none-any.whl (144 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m144.4/144.4 kB\u001b[0m \u001b[31m10.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading azure_mgmt_iothub-3.0.0-py3-none-any.whl (1.3 MB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m51.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading azure_mgmt_iothubprovisioningservices-1.1.0-py3-none-any.whl (52 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m52.5/52.5 kB\u001b[0m \u001b[31m4.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading azure_mgmt_keyvault-10.3.0-py3-none-any.whl (933 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m933.0/933.0 kB\u001b[0m \u001b[31m40.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading azure_mgmt_loganalytics-13.0.0b4-py3-none-any.whl (162 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m162.6/162.6 kB\u001b[0m \u001b[31m8.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading azure_mgmt_marketplaceordering-1.1.0-py2.py3-none-any.whl (26 kB)\n",
      "Downloading azure_mgmt_policyinsights-1.1.0b4-py3-none-any.whl (127 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m127.0/127.0 kB\u001b[0m \u001b[31m8.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading azure_mgmt_resource-23.1.1-py3-none-any.whl (2.6 MB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.6/2.6 MB\u001b[0m \u001b[31m66.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading azure_mgmt_security-6.0.0-py3-none-any.whl (1.1 MB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.1/1.1 MB\u001b[0m \u001b[31m49.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading azure_mgmt_servicefabricmanagedclusters-2.0.0b6-py3-none-any.whl (204 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m204.4/204.4 kB\u001b[0m \u001b[31m13.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading azure_mgmt_servicelinker-1.2.0b2-py3-none-any.whl (95 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m95.1/95.1 kB\u001b[0m \u001b[31m7.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading azure_mgmt_signalr-2.0.0b1-py3-none-any.whl (123 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m123.6/123.6 kB\u001b[0m \u001b[31m9.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading azure_mgmt_sql-4.0.0b19-py3-none-any.whl (1.2 MB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.2/1.2 MB\u001b[0m \u001b[31m19.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading azure_mgmt_sqlvirtualmachine-1.0.0b5-py3-none-any.whl (95 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m95.3/95.3 kB\u001b[0m \u001b[31m4.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading azure_mgmt_storage-21.2.0-py3-none-any.whl (3.2 MB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.2/3.2 MB\u001b[0m \u001b[31m62.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading azure_mgmt_synapse-2.1.0b5-py3-none-any.whl (547 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m547.1/547.1 kB\u001b[0m \u001b[31m33.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading azure_mgmt_web-7.2.0-py3-none-any.whl (6.0 MB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m6.0/6.0 MB\u001b[0m \u001b[31m65.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading azure_monitor_query-1.2.0-py3-none-any.whl (113 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m113.4/113.4 kB\u001b[0m \u001b[31m10.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading msal-1.31.0-py3-none-any.whl (113 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m113.1/113.1 kB\u001b[0m \u001b[31m10.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading msal_extensions-1.2.0-py3-none-any.whl (19 kB)\n",
      "Downloading semver-2.13.0-py2.py3-none-any.whl (12 kB)\n",
      "Downloading azure_cli_telemetry-1.1.0-py3-none-any.whl (11 kB)\n",
      "Downloading python_docx-1.1.2-py3-none-any.whl (244 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m244.3/244.3 kB\u001b[0m \u001b[31m20.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading antlr4_python3_runtime-4.13.2-py3-none-any.whl (144 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m144.5/144.5 kB\u001b[0m \u001b[31m12.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading azure_appconfiguration-1.7.1-py3-none-any.whl (90 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m91.0/91.0 kB\u001b[0m \u001b[31m6.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading azure_batch-14.2.0-py3-none-any.whl (243 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m243.3/243.3 kB\u001b[0m \u001b[31m18.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading azure_core-1.32.0-py3-none-any.whl (198 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m198.9/198.9 kB\u001b[0m \u001b[31m18.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading azure_cosmos-3.2.0-py2.py3-none-any.whl (106 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m106.6/106.6 kB\u001b[0m \u001b[31m9.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading azure_datalake_store-0.0.53-py2.py3-none-any.whl (55 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m55.3/55.3 kB\u001b[0m \u001b[31m4.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading azure_graphrbac-0.60.0-py2.py3-none-any.whl (139 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m139.7/139.7 kB\u001b[0m \u001b[31m9.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading azure_mgmt_applicationinsights-1.0.0-py2.py3-none-any.whl (302 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m303.0/303.0 kB\u001b[0m \u001b[31m20.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading azure_mgmt_authorization-4.0.0-py3-none-any.whl (1.1 MB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.1/1.1 MB\u001b[0m \u001b[31m47.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading azure_mgmt_batch-17.3.0-py3-none-any.whl (162 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m162.1/162.1 kB\u001b[0m \u001b[31m11.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading azure_mgmt_botservice-2.0.0-py3-none-any.whl (126 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m126.5/126.5 kB\u001b[0m \u001b[31m10.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading azure_mgmt_cognitiveservices-13.5.0-py3-none-any.whl (144 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m144.3/144.3 kB\u001b[0m \u001b[31m11.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading azure_mgmt_compute-33.0.0-py3-none-any.whl (5.9 MB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m5.9/5.9 MB\u001b[0m \u001b[31m64.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading azure_mgmt_containerservice-32.0.0-py3-none-any.whl (10.3 MB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m10.3/10.3 MB\u001b[0m \u001b[31m60.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading azure_mgmt_databoxedge-1.0.0-py2.py3-none-any.whl (1.3 MB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m42.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading azure_mgmt_datamigration-10.0.0-py2.py3-none-any.whl (174 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m174.5/174.5 kB\u001b[0m \u001b[31m12.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading azure_mgmt_devtestlabs-4.0.0-py2.py3-none-any.whl (137 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m137.0/137.0 kB\u001b[0m \u001b[31m10.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading azure_mgmt_dns-8.0.0-py2.py3-none-any.whl (118 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m118.9/118.9 kB\u001b[0m \u001b[31m9.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading azure_mgmt_eventhub-10.1.0-py3-none-any.whl (598 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m598.9/598.9 kB\u001b[0m \u001b[31m30.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading azure_mgmt_imagebuilder-1.3.0-py3-none-any.whl (78 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m78.1/78.1 kB\u001b[0m \u001b[31m5.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading azure_mgmt_iotcentral-10.0.0b2-py3-none-any.whl (55 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m55.4/55.4 kB\u001b[0m \u001b[31m3.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading azure_mgmt_kusto-0.3.0-py2.py3-none-any.whl (73 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m73.9/73.9 kB\u001b[0m \u001b[31m6.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading azure_mgmt_managementgroups-1.0.0-py2.py3-none-any.whl (58 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m58.1/58.1 kB\u001b[0m \u001b[31m3.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading azure_mgmt_maps-2.0.0-py2.py3-none-any.whl (38 kB)\n",
      "Downloading azure_mgmt_media-9.0.0-py3-none-any.whl (211 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m211.1/211.1 kB\u001b[0m \u001b[31m14.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading azure_mgmt_monitor-5.0.1-py3-none-any.whl (1.1 MB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.1/1.1 MB\u001b[0m \u001b[31m40.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading azure_mgmt_msi-7.0.0-py3-none-any.whl (183 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m183.4/183.4 kB\u001b[0m \u001b[31m12.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading azure_mgmt_netapp-10.1.0-py3-none-any.whl (200 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m200.7/200.7 kB\u001b[0m \u001b[31m16.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading azure_mgmt_privatedns-1.0.0-py2.py3-none-any.whl (43 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m44.0/44.0 kB\u001b[0m \u001b[31m2.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading azure_mgmt_rdbms-10.2.0b18-py3-none-any.whl (556 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m556.6/556.6 kB\u001b[0m \u001b[31m32.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading azure_mgmt_recoveryservices-3.0.0-py3-none-any.whl (108 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m108.6/108.6 kB\u001b[0m \u001b[31m9.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading azure_mgmt_recoveryservicesbackup-9.1.0-py3-none-any.whl (570 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m570.9/570.9 kB\u001b[0m \u001b[31m32.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading azure_mgmt_redhatopenshift-1.5.0-py3-none-any.whl (408 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m408.4/408.4 kB\u001b[0m \u001b[31m20.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading azure_mgmt_redis-14.4.0-py3-none-any.whl (127 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m127.4/127.4 kB\u001b[0m \u001b[31m8.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading azure_mgmt_search-9.1.0-py3-none-any.whl (110 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m110.3/110.3 kB\u001b[0m \u001b[31m10.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading azure_mgmt_servicebus-8.2.0-py3-none-any.whl (940 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m940.1/940.1 kB\u001b[0m \u001b[31m40.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading azure_mgmt_servicefabric-2.1.0-py3-none-any.whl (124 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m124.3/124.3 kB\u001b[0m \u001b[31m11.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading azure_mgmt_trafficmanager-1.0.0-py3-none-any.whl (49 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m49.3/49.3 kB\u001b[0m \u001b[31m3.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading azure_multiapi_storage-1.3.0-py2.py3-none-any.whl (2.9 MB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.9/2.9 MB\u001b[0m \u001b[31m50.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading azure_storage_blob-12.23.1-py3-none-any.whl (405 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m405.6/405.6 kB\u001b[0m \u001b[31m24.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading azure_storage_common-1.4.2-py2.py3-none-any.whl (47 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m47.4/47.4 kB\u001b[0m \u001b[31m3.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading azure_synapse_accesscontrol-0.5.0-py2.py3-none-any.whl (30 kB)\n",
      "Downloading azure_synapse_artifacts-0.19.0-py3-none-any.whl (495 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m495.8/495.8 kB\u001b[0m \u001b[31m33.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading azure_synapse_managedprivateendpoints-0.4.0-py2.py3-none-any.whl (45 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m46.0/46.0 kB\u001b[0m \u001b[31m3.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading azure_synapse_spark-0.2.0-py2.py3-none-any.whl (29 kB)\n",
      "Downloading colorama-0.4.6-py2.py3-none-any.whl (25 kB)\n",
      "Downloading elastic_transport-8.15.1-py3-none-any.whl (64 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m64.4/64.4 kB\u001b[0m \u001b[31m3.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading fabric-3.2.2-py3-none-any.whl (59 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m59.4/59.4 kB\u001b[0m \u001b[31m4.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading isodate-0.7.2-py3-none-any.whl (22 kB)\n",
      "Downloading javaproperties-0.5.2-py2.py3-none-any.whl (19 kB)\n",
      "Downloading jsondiff-2.0.0-py3-none-any.whl (6.6 kB)\n",
      "Downloading pycomposefile-0.0.32-py3-none-any.whl (28 kB)\n",
      "Downloading PyGithub-1.59.1-py3-none-any.whl (342 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m342.2/342.2 kB\u001b[0m \u001b[31m27.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading PyNaCl-1.5.0-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl (856 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m856.7/856.7 kB\u001b[0m \u001b[31m42.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading scp-0.13.6-py2.py3-none-any.whl (8.2 kB)\n",
      "Downloading sshtunnel-0.1.5-py2.py3-none-any.whl (23 kB)\n",
      "Downloading websocket_client-1.3.3-py3-none-any.whl (54 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m54.3/54.3 kB\u001b[0m \u001b[31m4.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading xmltodict-0.14.2-py2.py3-none-any.whl (10.0 kB)\n",
      "Downloading argcomplete-3.3.0-py3-none-any.whl (42 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m42.6/42.6 kB\u001b[0m \u001b[31m3.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading azure_common-1.1.28-py2.py3-none-any.whl (14 kB)\n",
      "Downloading azure_mgmt_core-1.5.0-py3-none-any.whl (30 kB)\n",
      "Downloading decorator-5.1.1-py3-none-any.whl (9.1 kB)\n",
      "Downloading humanfriendly-10.0-py2.py3-none-any.whl (86 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m86.8/86.8 kB\u001b[0m \u001b[31m6.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading invoke-2.2.0-py3-none-any.whl (160 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m160.3/160.3 kB\u001b[0m \u001b[31m13.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading knack-0.11.0-py3-none-any.whl (60 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m60.8/60.8 kB\u001b[0m \u001b[31m5.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading microsoft_security_utilities_secret_masker-1.0.0b3-py3-none-any.whl (16 kB)\n",
      "Downloading msrest-0.7.1-py3-none-any.whl (85 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m85.4/85.4 kB\u001b[0m \u001b[31m7.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading msrestazure-0.6.4.post1-py2.py3-none-any.whl (40 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m40.8/40.8 kB\u001b[0m \u001b[31m2.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading paramiko-3.5.0-py3-none-any.whl (227 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m227.1/227.1 kB\u001b[0m \u001b[31m18.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading pkginfo-1.11.2-py3-none-any.whl (31 kB)\n",
      "Downloading portalocker-2.10.1-py3-none-any.whl (18 kB)\n",
      "Downloading jmespath-1.0.1-py3-none-any.whl (20 kB)\n",
      "Downloading adal-1.2.7-py2.py3-none-any.whl (55 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m55.5/55.5 kB\u001b[0m \u001b[31m4.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading applicationinsights-0.11.10-py2.py3-none-any.whl (55 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m55.1/55.1 kB\u001b[0m \u001b[31m3.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hDownloading bcrypt-4.2.0-cp39-abi3-manylinux_2_28_x86_64.whl (273 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m273.8/273.8 kB\u001b[0m \u001b[31m20.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hInstalling collected packages: microsoft-security-utilities-secret-masker, jsondiff, azure-common, applicationinsights, antlr4-python3-runtime, xmltodict, websocket-client, semver, python-docx, pycomposefile, portalocker, pkginfo, jmespath, javaproperties, isodate, invoke, humanfriendly, elastic-transport, decorator, colorama, bcrypt, argcomplete, PyNaCl, knack, elasticsearch, azure-cosmos, azure-core, azure-cli-telemetry, paramiko, msrest, azure-storage-common, azure-storage-blob, azure-monitor-query, azure-mgmt-core, azure-keyvault-secrets, azure-keyvault-keys, azure-keyvault-certificates, azure-keyvault-administration, azure-appconfiguration, adal, sshtunnel, scp, PyGithub, msrestazure, msal, fabric, azure-synapse-spark, azure-synapse-managedprivateendpoints, azure-synapse-artifacts, azure-synapse-accesscontrol, azure-storage-file-datalake, azure-multiapi-storage, azure-mgmt-web, azure-mgmt-trafficmanager, azure-mgmt-synapse, azure-mgmt-storage, azure-mgmt-sqlvirtualmachine, azure-mgmt-sql, azure-mgmt-signalr, azure-mgmt-servicelinker, azure-mgmt-servicefabricmanagedclusters, azure-mgmt-servicefabric, azure-mgmt-servicebus, azure-mgmt-security, azure-mgmt-search, azure-mgmt-resource, azure-mgmt-redis, azure-mgmt-redhatopenshift, azure-mgmt-recoveryservicesbackup, azure-mgmt-recoveryservices, azure-mgmt-rdbms, azure-mgmt-privatedns, azure-mgmt-policyinsights, azure-mgmt-netapp, azure-mgmt-msi, azure-mgmt-monitor, azure-mgmt-media, azure-mgmt-marketplaceordering, azure-mgmt-maps, azure-mgmt-managementgroups, azure-mgmt-loganalytics, azure-mgmt-keyvault, azure-mgmt-iothubprovisioningservices, azure-mgmt-iothub, azure-mgmt-iotcentral, azure-mgmt-imagebuilder, azure-mgmt-hdinsight, azure-mgmt-extendedlocation, azure-mgmt-eventhub, azure-mgmt-eventgrid, azure-mgmt-dns, azure-mgmt-datamigration, azure-mgmt-databoxedge, azure-mgmt-cosmosdb, azure-mgmt-containerservice, azure-mgmt-containerregistry, azure-mgmt-containerinstance, azure-mgmt-compute, azure-mgmt-cognitiveservices, azure-mgmt-cdn, azure-mgmt-botservice, azure-mgmt-billing, azure-mgmt-batchai, azure-mgmt-batch, azure-mgmt-authorization, azure-mgmt-applicationinsights, azure-mgmt-appcontainers, azure-mgmt-appconfiguration, azure-mgmt-apimanagement, azure-mgmt-advisor, azure-data-tables, msal-extensions, azure-mgmt-kusto, azure-mgmt-devtestlabs, azure-graphrbac, azure-datalake-store, azure-batch, azure-identity, azure-cli-core, azure-cli\n",
      "  Attempting uninstall: websocket-client\n",
      "    Found existing installation: websocket-client 1.8.0\n",
      "    Uninstalling websocket-client-1.8.0:\n",
      "      Successfully uninstalled websocket-client-1.8.0\n",
      "  Attempting uninstall: decorator\n",
      "    Found existing installation: decorator 4.4.2\n",
      "    Uninstalling decorator-4.4.2:\n",
      "      Successfully uninstalled decorator-4.4.2\n",
      "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
      "ipython 7.34.0 requires jedi>=0.16, which is not installed.\n",
      "moviepy 1.0.3 requires decorator<5.0,>=4.0.2, but you have decorator 5.1.1 which is incompatible.\u001b[0m\u001b[31m\n",
      "\u001b[0mSuccessfully installed PyGithub-1.59.1 PyNaCl-1.5.0 adal-1.2.7 antlr4-python3-runtime-4.13.2 applicationinsights-0.11.10 argcomplete-3.3.0 azure-appconfiguration-1.7.1 azure-batch-14.2.0 azure-cli-2.65.0 azure-cli-core-2.65.0 azure-cli-telemetry-1.1.0 azure-common-1.1.28 azure-core-1.32.0 azure-cosmos-3.2.0 azure-data-tables-12.4.0 azure-datalake-store-0.0.53 azure-graphrbac-0.60.0 azure-identity-1.19.0 azure-keyvault-administration-4.4.0b2 azure-keyvault-certificates-4.7.0 azure-keyvault-keys-4.9.0b3 azure-keyvault-secrets-4.7.0 azure-mgmt-advisor-9.0.0 azure-mgmt-apimanagement-4.0.0 azure-mgmt-appconfiguration-3.0.0 azure-mgmt-appcontainers-2.0.0 azure-mgmt-applicationinsights-1.0.0 azure-mgmt-authorization-4.0.0 azure-mgmt-batch-17.3.0 azure-mgmt-batchai-7.0.0b1 azure-mgmt-billing-6.0.0 azure-mgmt-botservice-2.0.0 azure-mgmt-cdn-12.0.0 azure-mgmt-cognitiveservices-13.5.0 azure-mgmt-compute-33.0.0 azure-mgmt-containerinstance-10.1.0 azure-mgmt-containerregistry-10.3.0 azure-mgmt-containerservice-32.0.0 azure-mgmt-core-1.5.0 azure-mgmt-cosmosdb-9.6.0 azure-mgmt-databoxedge-1.0.0 azure-mgmt-datamigration-10.0.0 azure-mgmt-devtestlabs-4.0.0 azure-mgmt-dns-8.0.0 azure-mgmt-eventgrid-10.2.0b2 azure-mgmt-eventhub-10.1.0 azure-mgmt-extendedlocation-1.0.0b2 azure-mgmt-hdinsight-9.0.0b3 azure-mgmt-imagebuilder-1.3.0 azure-mgmt-iotcentral-10.0.0b2 azure-mgmt-iothub-3.0.0 azure-mgmt-iothubprovisioningservices-1.1.0 azure-mgmt-keyvault-10.3.0 azure-mgmt-kusto-0.3.0 azure-mgmt-loganalytics-13.0.0b4 azure-mgmt-managementgroups-1.0.0 azure-mgmt-maps-2.0.0 azure-mgmt-marketplaceordering-1.1.0 azure-mgmt-media-9.0.0 azure-mgmt-monitor-5.0.1 azure-mgmt-msi-7.0.0 azure-mgmt-netapp-10.1.0 azure-mgmt-policyinsights-1.1.0b4 azure-mgmt-privatedns-1.0.0 azure-mgmt-rdbms-10.2.0b18 azure-mgmt-recoveryservices-3.0.0 azure-mgmt-recoveryservicesbackup-9.1.0 azure-mgmt-redhatopenshift-1.5.0 azure-mgmt-redis-14.4.0 azure-mgmt-resource-23.1.1 azure-mgmt-search-9.1.0 azure-mgmt-security-6.0.0 azure-mgmt-servicebus-8.2.0 azure-mgmt-servicefabric-2.1.0 azure-mgmt-servicefabricmanagedclusters-2.0.0b6 azure-mgmt-servicelinker-1.2.0b2 azure-mgmt-signalr-2.0.0b1 azure-mgmt-sql-4.0.0b19 azure-mgmt-sqlvirtualmachine-1.0.0b5 azure-mgmt-storage-21.2.0 azure-mgmt-synapse-2.1.0b5 azure-mgmt-trafficmanager-1.0.0 azure-mgmt-web-7.2.0 azure-monitor-query-1.2.0 azure-multiapi-storage-1.3.0 azure-storage-blob-12.23.1 azure-storage-common-1.4.2 azure-storage-file-datalake-12.17.0 azure-synapse-accesscontrol-0.5.0 azure-synapse-artifacts-0.19.0 azure-synapse-managedprivateendpoints-0.4.0 azure-synapse-spark-0.2.0 bcrypt-4.2.0 colorama-0.4.6 decorator-5.1.1 elastic-transport-8.15.1 elasticsearch-8.14.0 fabric-3.2.2 humanfriendly-10.0 invoke-2.2.0 isodate-0.7.2 javaproperties-0.5.2 jmespath-1.0.1 jsondiff-2.0.0 knack-0.11.0 microsoft-security-utilities-secret-masker-1.0.0b3 msal-1.31.0 msal-extensions-1.2.0 msrest-0.7.1 msrestazure-0.6.4.post1 paramiko-3.5.0 pkginfo-1.11.2 portalocker-2.10.1 pycomposefile-0.0.32 python-docx-1.1.2 scp-0.13.6 semver-2.13.0 sshtunnel-0.1.5 websocket-client-1.3.3 xmltodict-0.14.2\n"
     ]
    }
   ],
   "source": [
    "!python3 -m pip install elasticsearch==8.14 azure-identity azure-storage-file-datalake azure-cli python-docx\n",
    "\n",
    "import os\n",
    "import csv\n",
    "import chardet\n",
    "\n",
    "from docx import Document\n",
    "from io import StringIO\n",
    "from getpass import getpass\n",
    "from google.colab import files\n",
    "from elasticsearch import Elasticsearch, exceptions, helpers\n",
    "from elasticsearch.helpers import bulk\n",
    "from azure.identity import DefaultAzureCredential\n",
    "from azure.storage.filedatalake import DataLakeDirectoryClient, DataLakeServiceClient"
   ]
  },
  {
   "cell_type": "markdown",
   "source": [
    "Let's sign in with your Microsoft account. At the end of the process, the Google Colab console will display your available subscriptions. Select your subscription by entering the subscription number shown in the console."
   ],
   "metadata": {
    "id": "Ur5tGmKkjtTT"
   }
  },
  {
   "cell_type": "code",
   "source": [
    "!az login --allow-no-subscriptions # For no subscriptions login"
   ],
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "3tLzCaZnKI68",
    "outputId": "6303c2a3-2fd5-4e0c-cf22-00e867a9b879"
   },
   "execution_count": null,
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "\u001b[91mError loading command module 'mysql': cannot import name 'mysql_flexibleservers' from 'azure.mgmt.rdbms' (/usr/local/lib/python3.10/dist-packages/azure/mgmt/rdbms/__init__.py)\u001b[0m\n",
      "\u001b[91mError loading command module 'rdbms': No module named 'azure.mgmt.rdbms.mysql_flexibleservers'\u001b[0m\n",
      "\u001b[93mTo sign in, use a web browser to open the page https://microsoft.com/devicelogin and enter the code AYGZ24EJG to authenticate.\u001b[0m\n",
      "\n",
      "Retrieving tenants and subscriptions for the selection...\n",
      "\u001b[93mThe following tenants don't contain accessible subscriptions. Use `az login --allow-no-subscriptions` to have tenant level access.\u001b[0m\n",
      "\u001b[93m56783f2b-514d-4ff9-b3ab-3fdc5e7ace5d 'Kibernum'\u001b[0m\n",
      "\n",
      "[Tenant and subscription selection]\n",
      "\n",
      "No     Subscription name          Subscription ID                       Tenant\n",
      "-----  -------------------------  ------------------------------------  ------------------------------------\n",
      "\u001b[96m[1]\u001b[0m *  \u001b[96mN/A(tenant level account)\u001b[0m  \u001b[96m56783f2b-514d-4ff9-b3ab-3fdc5e7ace5d\u001b[0m  \u001b[96m56783f2b-514d-4ff9-b3ab-3fdc5e7ace5d\u001b[0m\n",
      "\n",
      "The default is marked with an *; the default tenant is '56783f2b-514d-4ff9-b3ab-3fdc5e7ace5d' and subscription is 'N/A(tenant level account)' (56783f2b-514d-4ff9-b3ab-3fdc5e7ace5d).\n",
      "\n",
      "Select a subscription and tenant (Type a number or Enter for no changes): 1\n",
      "\n",
      "Tenant: 56783f2b-514d-4ff9-b3ab-3fdc5e7ace5d\n",
      "Subscription: N/A(tenant level account) (56783f2b-514d-4ff9-b3ab-3fdc5e7ace5d)\n",
      "\n",
      "[Announcements]\n",
      "With the new Azure CLI login experience, you can select the subscription you want to use more easily. Learn more about it and its configuration at https://go.microsoft.com/fwlink/?linkid=2271236\n",
      "\n",
      "If you encounter any problem, please open an issue at https://aka.ms/azclibug\n",
      "\n",
      "\u001b[93m[Warning] The login output has been updated. Please be aware that it no longer displays the full list of available subscriptions by default.\n",
      "\u001b[0m\n"
     ]
    }
   ]
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Variable Declaration\n",
    "\n",
    "Necessary variables for the operation of the services we will use. Please, insert your Elasticsearch credentials below:"
   ],
   "metadata": {
    "id": "0IlxbM5skilk"
   }
  },
  {
   "cell_type": "code",
   "source": [
    "ELASTIC_CLUSTER_ID = getpass(\"Elastic Cloud ID: \")\n",
    "ELASTIC_API_KEY = getpass(\"Elastic Api Key: \")\n",
    "\n",
    "ONELAKE_ACCOUNT_NAME = \"onelake\"\n",
    "ONELAKE_WORKSPACE_NAME = \"ShoesticWorkspace\"\n",
    "# Path in format <DataLake>.Lakehouse/files/<Folder path>\n",
    "ONELAKE_DATA_PATH = \"shoesticDatalake.Lakehouse/Files/ProductsData\""
   ],
   "metadata": {
    "id": "zFoRfKBBl6iT",
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "outputId": "0eae3aff-bfd9-442b-ae5a-73fa146c6fc7"
   },
   "execution_count": null,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Elastic Cloud ID: ··········\n",
      "Elastic Api Key: ··········\n"
     ]
    }
   ]
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Initialization of OneLake and Elasticsearch Services"
   ],
   "metadata": {
    "id": "w4DGPSxrmb6P"
   }
  },
  {
   "cell_type": "code",
   "source": [
    "# Microsoft token\n",
    "token_credential = DefaultAzureCredential()\n",
    "\n",
    "# OneLake services\n",
    "service_client = DataLakeServiceClient(\n",
    "    account_url=f\"https://{ONELAKE_ACCOUNT_NAME}.dfs.fabric.microsoft.com\",\n",
    "    credential=token_credential,\n",
    ")\n",
    "file_system_client = service_client.get_file_system_client(ONELAKE_WORKSPACE_NAME)\n",
    "directory_client = file_system_client.get_directory_client(ONELAKE_DATA_PATH)\n",
    "\n",
    "# Elasticsearch client\n",
    "es_client = Elasticsearch(\n",
    "    cloud_id=ELASTIC_CLUSTER_ID,\n",
    "    api_key=ELASTIC_API_KEY,\n",
    ")"
   ],
   "metadata": {
    "id": "rU9db1cJQDS3"
   },
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Functions declaration"
   ],
   "metadata": {
    "id": "r1emsb8hmwsb"
   }
  },
  {
   "cell_type": "code",
   "source": [
    "# Upload a file to a LakeHouse directory\n",
    "def upload_file_to_directory(directory_client, local_path, file_name):\n",
    "    file_client = directory_client.get_file_client(file_name)\n",
    "\n",
    "    with open(local_path, mode=\"rb\") as data:\n",
    "        file_client.upload_data(data, overwrite=True)\n",
    "\n",
    "    print(f\"File: {file_name} uploaded to the data lake.\")"
   ],
   "metadata": {
    "id": "dVc_U30vm0L4"
   },
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    "# Get directory contents from your lake folder\n",
    "def list_directory_contents(file_system_client, directory_name):\n",
    "    paths = file_system_client.get_paths(path=directory_name)\n",
    "\n",
    "    for path in paths:\n",
    "        print(path.name + \"\\n\")"
   ],
   "metadata": {
    "id": "LNwweKTAntot"
   },
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    "# Get a file by name from your lake folder\n",
    "def get_file_by_name(file_name, directory_client):\n",
    "    return directory_client.get_file_client(file_name)"
   ],
   "metadata": {
    "id": "zBS6j7bVQNcY"
   },
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    "# Google Colab saves the files in a internal folder, to prevent the automatic rename of the files, we must to clean de directory\n",
    "def clean_directory(directory):\n",
    "    for filename in os.listdir(directory):\n",
    "        file_path = os.path.join(directory, filename)\n",
    "        if os.path.isfile(file_path):\n",
    "            os.remove(file_path)\n",
    "            print(f\"File '{filename}' removed successfully.\")"
   ],
   "metadata": {
    "id": "kuW-D3nQ6mX0"
   },
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    "# Decode csv\n",
    "def get_csv_content(file_client):\n",
    "    download = file_client.download_file()\n",
    "    file_content = download.readall()\n",
    "\n",
    "    result = chardet.detect(file_content)\n",
    "    encoding = result[\"encoding\"]\n",
    "\n",
    "    return file_content.decode(encoding)"
   ],
   "metadata": {
    "id": "eGLBygpK9YTK"
   },
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    "# Decode docx\n",
    "def get_docx_content(file_client):\n",
    "    download = file_client.download_file()\n",
    "    file_content = download.readall()\n",
    "\n",
    "    temp_file_path = \"temp.docx\"\n",
    "    with open(temp_file_path, \"wb\") as temp_file:\n",
    "        temp_file.write(file_content)\n",
    "\n",
    "    doc = Document(temp_file_path)\n",
    "    text = []\n",
    "    for paragraph in doc.paragraphs:\n",
    "        text.append(paragraph.text)\n",
    "\n",
    "    return \"\\n\".join(text)"
   ],
   "metadata": {
    "id": "FelmeP4l8hQP"
   },
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Uploading data to OneLake"
   ],
   "metadata": {
    "id": "G7U09Dd3_fy7"
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "Before start to upload your files is higlhy recommended to clean the Google Colab directory."
   ],
   "metadata": {
    "id": "luDALXfWCBQI"
   }
  },
  {
   "cell_type": "code",
   "source": [
    "clean_directory(\"/content\")"
   ],
   "metadata": {
    "id": "tEi6Ls4qB_OZ"
   },
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "source": [
    "Upload the csv file:"
   ],
   "metadata": {
    "id": "yQizi7s_CKQw"
   }
  },
  {
   "cell_type": "code",
   "source": [
    "# Upload csv to Colab\n",
    "\n",
    "# Upload one local file\n",
    "uploaded = files.upload()\n",
    "csv_file_name = list(uploaded.keys())[0]\n",
    "\n",
    "# Google Colab file path\n",
    "csv_local_path = f\"/content/{csv_file_name}\""
   ],
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 72
    },
    "id": "K0BWR3VitW1n",
    "outputId": "75bbe968-0ed8-4e58-d18e-7b14bef7a0d4"
   },
   "execution_count": null,
   "outputs": [
    {
     "output_type": "display_data",
     "data": {
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ],
      "text/html": [
       "\n",
       "     <input type=\"file\" id=\"files-43564288-65e0-4044-8ea2-289e80ef8381\" name=\"files[]\" multiple disabled\n",
       "        style=\"border:none\" />\n",
       "     <output id=\"result-43564288-65e0-4044-8ea2-289e80ef8381\">\n",
       "      Upload widget is only available when the cell has been executed in the\n",
       "      current browser session. Please rerun this cell to enable.\n",
       "      </output>\n",
       "      <script>// Copyright 2017 Google LLC\n",
       "//\n",
       "// Licensed under the Apache License, Version 2.0 (the \"License\");\n",
       "// you may not use this file except in compliance with the License.\n",
       "// You may obtain a copy of the License at\n",
       "//\n",
       "//      http://www.apache.org/licenses/LICENSE-2.0\n",
       "//\n",
       "// Unless required by applicable law or agreed to in writing, software\n",
       "// distributed under the License is distributed on an \"AS IS\" BASIS,\n",
       "// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
       "// See the License for the specific language governing permissions and\n",
       "// limitations under the License.\n",
       "\n",
       "/**\n",
       " * @fileoverview Helpers for google.colab Python module.\n",
       " */\n",
       "(function(scope) {\n",
       "function span(text, styleAttributes = {}) {\n",
       "  const element = document.createElement('span');\n",
       "  element.textContent = text;\n",
       "  for (const key of Object.keys(styleAttributes)) {\n",
       "    element.style[key] = styleAttributes[key];\n",
       "  }\n",
       "  return element;\n",
       "}\n",
       "\n",
       "// Max number of bytes which will be uploaded at a time.\n",
       "const MAX_PAYLOAD_SIZE = 100 * 1024;\n",
       "\n",
       "function _uploadFiles(inputId, outputId) {\n",
       "  const steps = uploadFilesStep(inputId, outputId);\n",
       "  const outputElement = document.getElementById(outputId);\n",
       "  // Cache steps on the outputElement to make it available for the next call\n",
       "  // to uploadFilesContinue from Python.\n",
       "  outputElement.steps = steps;\n",
       "\n",
       "  return _uploadFilesContinue(outputId);\n",
       "}\n",
       "\n",
       "// This is roughly an async generator (not supported in the browser yet),\n",
       "// where there are multiple asynchronous steps and the Python side is going\n",
       "// to poll for completion of each step.\n",
       "// This uses a Promise to block the python side on completion of each step,\n",
       "// then passes the result of the previous step as the input to the next step.\n",
       "function _uploadFilesContinue(outputId) {\n",
       "  const outputElement = document.getElementById(outputId);\n",
       "  const steps = outputElement.steps;\n",
       "\n",
       "  const next = steps.next(outputElement.lastPromiseValue);\n",
       "  return Promise.resolve(next.value.promise).then((value) => {\n",
       "    // Cache the last promise value to make it available to the next\n",
       "    // step of the generator.\n",
       "    outputElement.lastPromiseValue = value;\n",
       "    return next.value.response;\n",
       "  });\n",
       "}\n",
       "\n",
       "/**\n",
       " * Generator function which is called between each async step of the upload\n",
       " * process.\n",
       " * @param {string} inputId Element ID of the input file picker element.\n",
       " * @param {string} outputId Element ID of the output display.\n",
       " * @return {!Iterable<!Object>} Iterable of next steps.\n",
       " */\n",
       "function* uploadFilesStep(inputId, outputId) {\n",
       "  const inputElement = document.getElementById(inputId);\n",
       "  inputElement.disabled = false;\n",
       "\n",
       "  const outputElement = document.getElementById(outputId);\n",
       "  outputElement.innerHTML = '';\n",
       "\n",
       "  const pickedPromise = new Promise((resolve) => {\n",
       "    inputElement.addEventListener('change', (e) => {\n",
       "      resolve(e.target.files);\n",
       "    });\n",
       "  });\n",
       "\n",
       "  const cancel = document.createElement('button');\n",
       "  inputElement.parentElement.appendChild(cancel);\n",
       "  cancel.textContent = 'Cancel upload';\n",
       "  const cancelPromise = new Promise((resolve) => {\n",
       "    cancel.onclick = () => {\n",
       "      resolve(null);\n",
       "    };\n",
       "  });\n",
       "\n",
       "  // Wait for the user to pick the files.\n",
       "  const files = yield {\n",
       "    promise: Promise.race([pickedPromise, cancelPromise]),\n",
       "    response: {\n",
       "      action: 'starting',\n",
       "    }\n",
       "  };\n",
       "\n",
       "  cancel.remove();\n",
       "\n",
       "  // Disable the input element since further picks are not allowed.\n",
       "  inputElement.disabled = true;\n",
       "\n",
       "  if (!files) {\n",
       "    return {\n",
       "      response: {\n",
       "        action: 'complete',\n",
       "      }\n",
       "    };\n",
       "  }\n",
       "\n",
       "  for (const file of files) {\n",
       "    const li = document.createElement('li');\n",
       "    li.append(span(file.name, {fontWeight: 'bold'}));\n",
       "    li.append(span(\n",
       "        `(${file.type || 'n/a'}) - ${file.size} bytes, ` +\n",
       "        `last modified: ${\n",
       "            file.lastModifiedDate ? file.lastModifiedDate.toLocaleDateString() :\n",
       "                                    'n/a'} - `));\n",
       "    const percent = span('0% done');\n",
       "    li.appendChild(percent);\n",
       "\n",
       "    outputElement.appendChild(li);\n",
       "\n",
       "    const fileDataPromise = new Promise((resolve) => {\n",
       "      const reader = new FileReader();\n",
       "      reader.onload = (e) => {\n",
       "        resolve(e.target.result);\n",
       "      };\n",
       "      reader.readAsArrayBuffer(file);\n",
       "    });\n",
       "    // Wait for the data to be ready.\n",
       "    let fileData = yield {\n",
       "      promise: fileDataPromise,\n",
       "      response: {\n",
       "        action: 'continue',\n",
       "      }\n",
       "    };\n",
       "\n",
       "    // Use a chunked sending to avoid message size limits. See b/62115660.\n",
       "    let position = 0;\n",
       "    do {\n",
       "      const length = Math.min(fileData.byteLength - position, MAX_PAYLOAD_SIZE);\n",
       "      const chunk = new Uint8Array(fileData, position, length);\n",
       "      position += length;\n",
       "\n",
       "      const base64 = btoa(String.fromCharCode.apply(null, chunk));\n",
       "      yield {\n",
       "        response: {\n",
       "          action: 'append',\n",
       "          file: file.name,\n",
       "          data: base64,\n",
       "        },\n",
       "      };\n",
       "\n",
       "      let percentDone = fileData.byteLength === 0 ?\n",
       "          100 :\n",
       "          Math.round((position / fileData.byteLength) * 100);\n",
       "      percent.textContent = `${percentDone}% done`;\n",
       "\n",
       "    } while (position < fileData.byteLength);\n",
       "  }\n",
       "\n",
       "  // All done.\n",
       "  yield {\n",
       "    response: {\n",
       "      action: 'complete',\n",
       "    }\n",
       "  };\n",
       "}\n",
       "\n",
       "scope.google = scope.google || {};\n",
       "scope.google.colab = scope.google.colab || {};\n",
       "scope.google.colab._files = {\n",
       "  _uploadFiles,\n",
       "  _uploadFilesContinue,\n",
       "};\n",
       "})(self);\n",
       "</script> "
      ]
     },
     "metadata": {}
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Saving products.csv to products.csv\n"
     ]
    }
   ]
  },
  {
   "cell_type": "markdown",
   "source": [
    "Upload the 3 docx files:"
   ],
   "metadata": {
    "id": "VgoWeLuU1rPJ"
   }
  },
  {
   "cell_type": "code",
   "source": [
    "# Upload docx files to Colab\n",
    "\n",
    "uploaded = files.upload()\n",
    "docx_files = list(uploaded.keys())\n",
    "\n",
    "docx_local_paths = [f\"/content/{file_name}\" for file_name in docx_files]"
   ],
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 195
    },
    "id": "KKNC_8t8BiTX",
    "outputId": "7bfb2715-b18f-49e3-fe81-72e78e24531f"
   },
   "execution_count": null,
   "outputs": [
    {
     "output_type": "display_data",
     "data": {
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ],
      "text/html": [
       "\n",
       "     <input type=\"file\" id=\"files-2c547b64-25bc-472d-b78a-9255399a475d\" name=\"files[]\" multiple disabled\n",
       "        style=\"border:none\" />\n",
       "     <output id=\"result-2c547b64-25bc-472d-b78a-9255399a475d\">\n",
       "      Upload widget is only available when the cell has been executed in the\n",
       "      current browser session. Please rerun this cell to enable.\n",
       "      </output>\n",
       "      <script>// Copyright 2017 Google LLC\n",
       "//\n",
       "// Licensed under the Apache License, Version 2.0 (the \"License\");\n",
       "// you may not use this file except in compliance with the License.\n",
       "// You may obtain a copy of the License at\n",
       "//\n",
       "//      http://www.apache.org/licenses/LICENSE-2.0\n",
       "//\n",
       "// Unless required by applicable law or agreed to in writing, software\n",
       "// distributed under the License is distributed on an \"AS IS\" BASIS,\n",
       "// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
       "// See the License for the specific language governing permissions and\n",
       "// limitations under the License.\n",
       "\n",
       "/**\n",
       " * @fileoverview Helpers for google.colab Python module.\n",
       " */\n",
       "(function(scope) {\n",
       "function span(text, styleAttributes = {}) {\n",
       "  const element = document.createElement('span');\n",
       "  element.textContent = text;\n",
       "  for (const key of Object.keys(styleAttributes)) {\n",
       "    element.style[key] = styleAttributes[key];\n",
       "  }\n",
       "  return element;\n",
       "}\n",
       "\n",
       "// Max number of bytes which will be uploaded at a time.\n",
       "const MAX_PAYLOAD_SIZE = 100 * 1024;\n",
       "\n",
       "function _uploadFiles(inputId, outputId) {\n",
       "  const steps = uploadFilesStep(inputId, outputId);\n",
       "  const outputElement = document.getElementById(outputId);\n",
       "  // Cache steps on the outputElement to make it available for the next call\n",
       "  // to uploadFilesContinue from Python.\n",
       "  outputElement.steps = steps;\n",
       "\n",
       "  return _uploadFilesContinue(outputId);\n",
       "}\n",
       "\n",
       "// This is roughly an async generator (not supported in the browser yet),\n",
       "// where there are multiple asynchronous steps and the Python side is going\n",
       "// to poll for completion of each step.\n",
       "// This uses a Promise to block the python side on completion of each step,\n",
       "// then passes the result of the previous step as the input to the next step.\n",
       "function _uploadFilesContinue(outputId) {\n",
       "  const outputElement = document.getElementById(outputId);\n",
       "  const steps = outputElement.steps;\n",
       "\n",
       "  const next = steps.next(outputElement.lastPromiseValue);\n",
       "  return Promise.resolve(next.value.promise).then((value) => {\n",
       "    // Cache the last promise value to make it available to the next\n",
       "    // step of the generator.\n",
       "    outputElement.lastPromiseValue = value;\n",
       "    return next.value.response;\n",
       "  });\n",
       "}\n",
       "\n",
       "/**\n",
       " * Generator function which is called between each async step of the upload\n",
       " * process.\n",
       " * @param {string} inputId Element ID of the input file picker element.\n",
       " * @param {string} outputId Element ID of the output display.\n",
       " * @return {!Iterable<!Object>} Iterable of next steps.\n",
       " */\n",
       "function* uploadFilesStep(inputId, outputId) {\n",
       "  const inputElement = document.getElementById(inputId);\n",
       "  inputElement.disabled = false;\n",
       "\n",
       "  const outputElement = document.getElementById(outputId);\n",
       "  outputElement.innerHTML = '';\n",
       "\n",
       "  const pickedPromise = new Promise((resolve) => {\n",
       "    inputElement.addEventListener('change', (e) => {\n",
       "      resolve(e.target.files);\n",
       "    });\n",
       "  });\n",
       "\n",
       "  const cancel = document.createElement('button');\n",
       "  inputElement.parentElement.appendChild(cancel);\n",
       "  cancel.textContent = 'Cancel upload';\n",
       "  const cancelPromise = new Promise((resolve) => {\n",
       "    cancel.onclick = () => {\n",
       "      resolve(null);\n",
       "    };\n",
       "  });\n",
       "\n",
       "  // Wait for the user to pick the files.\n",
       "  const files = yield {\n",
       "    promise: Promise.race([pickedPromise, cancelPromise]),\n",
       "    response: {\n",
       "      action: 'starting',\n",
       "    }\n",
       "  };\n",
       "\n",
       "  cancel.remove();\n",
       "\n",
       "  // Disable the input element since further picks are not allowed.\n",
       "  inputElement.disabled = true;\n",
       "\n",
       "  if (!files) {\n",
       "    return {\n",
       "      response: {\n",
       "        action: 'complete',\n",
       "      }\n",
       "    };\n",
       "  }\n",
       "\n",
       "  for (const file of files) {\n",
       "    const li = document.createElement('li');\n",
       "    li.append(span(file.name, {fontWeight: 'bold'}));\n",
       "    li.append(span(\n",
       "        `(${file.type || 'n/a'}) - ${file.size} bytes, ` +\n",
       "        `last modified: ${\n",
       "            file.lastModifiedDate ? file.lastModifiedDate.toLocaleDateString() :\n",
       "                                    'n/a'} - `));\n",
       "    const percent = span('0% done');\n",
       "    li.appendChild(percent);\n",
       "\n",
       "    outputElement.appendChild(li);\n",
       "\n",
       "    const fileDataPromise = new Promise((resolve) => {\n",
       "      const reader = new FileReader();\n",
       "      reader.onload = (e) => {\n",
       "        resolve(e.target.result);\n",
       "      };\n",
       "      reader.readAsArrayBuffer(file);\n",
       "    });\n",
       "    // Wait for the data to be ready.\n",
       "    let fileData = yield {\n",
       "      promise: fileDataPromise,\n",
       "      response: {\n",
       "        action: 'continue',\n",
       "      }\n",
       "    };\n",
       "\n",
       "    // Use a chunked sending to avoid message size limits. See b/62115660.\n",
       "    let position = 0;\n",
       "    do {\n",
       "      const length = Math.min(fileData.byteLength - position, MAX_PAYLOAD_SIZE);\n",
       "      const chunk = new Uint8Array(fileData, position, length);\n",
       "      position += length;\n",
       "\n",
       "      const base64 = btoa(String.fromCharCode.apply(null, chunk));\n",
       "      yield {\n",
       "        response: {\n",
       "          action: 'append',\n",
       "          file: file.name,\n",
       "          data: base64,\n",
       "        },\n",
       "      };\n",
       "\n",
       "      let percentDone = fileData.byteLength === 0 ?\n",
       "          100 :\n",
       "          Math.round((position / fileData.byteLength) * 100);\n",
       "      percent.textContent = `${percentDone}% done`;\n",
       "\n",
       "    } while (position < fileData.byteLength);\n",
       "  }\n",
       "\n",
       "  // All done.\n",
       "  yield {\n",
       "    response: {\n",
       "      action: 'complete',\n",
       "    }\n",
       "  };\n",
       "}\n",
       "\n",
       "scope.google = scope.google || {};\n",
       "scope.google.colab = scope.google.colab || {};\n",
       "scope.google.colab._files = {\n",
       "  _uploadFiles,\n",
       "  _uploadFilesContinue,\n",
       "};\n",
       "})(self);\n",
       "</script> "
      ]
     },
     "metadata": {}
    },
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Saving sport-sneakers.docx to sport-sneakers.docx\n",
      "Saving classic-loafers.docx to classic-loafers.docx\n",
      "Saving beach-flip-flops.docx to beach-flip-flops.docx\n"
     ]
    }
   ]
  },
  {
   "cell_type": "code",
   "source": [
    "# Upload files to Lakehouse\n",
    "upload_file_to_directory(directory_client, csv_local_path, csv_file_name)\n",
    "\n",
    "for docx_local_path in docx_local_paths:\n",
    "    docx_file_name = docx_local_path.split(\"/\")[-1]\n",
    "    upload_file_to_directory(directory_client, docx_local_path, docx_file_name)"
   ],
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "9u8tte8fuR6o",
    "outputId": "b13c059d-7452-4229-e346-080ed9d8b5f5"
   },
   "execution_count": null,
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "File: products.csv uploaded to the data lake.\n",
      "File: sport-sneakers.docx uploaded to the data lake.\n",
      "File: classic-loafers.docx uploaded to the data lake.\n",
      "File: beach-flip-flops.docx uploaded to the data lake.\n"
     ]
    }
   ]
  },
  {
   "cell_type": "markdown",
   "source": [
    "Check if files are now available in your datalake."
   ],
   "metadata": {
    "id": "a4tkLjo6A6yt"
   }
  },
  {
   "cell_type": "code",
   "source": [
    "list_directory_contents(file_system_client, ONELAKE_DATA_PATH)"
   ],
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "IipcuVFBGfbD",
    "outputId": "87cf4b1f-7bf7-4671-c12e-da221511b2dd"
   },
   "execution_count": null,
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "shoesticDatalake.Lakehouse/Files/ProductsData/beach-flip-flops.docx\n",
      "\n",
      "shoesticDatalake.Lakehouse/Files/ProductsData/classic-loafers.docx\n",
      "\n",
      "shoesticDatalake.Lakehouse/Files/ProductsData/products.csv\n",
      "\n",
      "shoesticDatalake.Lakehouse/Files/ProductsData/sport-sneakers.docx\n",
      "\n"
     ]
    }
   ]
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Indexing data to Elasticsearch"
   ],
   "metadata": {
    "id": "oTkmZC1w_qa7"
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Creating embeddings endpoint"
   ],
   "metadata": {
    "id": "LcRrrOwNZLWz"
   }
  },
  {
   "cell_type": "code",
   "source": [
    "try:\n",
    "    es_client.options(\n",
    "        request_timeout=60, max_retries=3, retry_on_timeout=True\n",
    "    ).inference.put_model(\n",
    "        inference_id=\"onelake-inference-endpoint\",\n",
    "        body={\n",
    "            \"task_type\": \"sparse_embedding\",\n",
    "            \"service\": \"elser\",\n",
    "            \"service_settings\": {\n",
    "                \"model_id\": \".elser_model_2_linux-x86_64\",\n",
    "                \"num_allocations\": 1,\n",
    "                \"num_threads\": 1,\n",
    "            },\n",
    "        },\n",
    "    )\n",
    "\n",
    "    print(\"Embedding endpoint created successfully.\")\n",
    "except exceptions.BadRequestError as e:\n",
    "    raise e"
   ],
   "metadata": {
    "id": "QFLH_4v6ZIM7",
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "outputId": "38f70e08-c7cb-4e82-dc1f-6076b9fb1802"
   },
   "execution_count": null,
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Embedding endpoint created successfully.\n"
     ]
    }
   ]
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Creating Mappings"
   ],
   "metadata": {
    "id": "s6ko5zhJZgKD"
   }
  },
  {
   "cell_type": "code",
   "source": [
    "# For data in csv\n",
    "try:\n",
    "    es_client.indices.create(\n",
    "        index=\"shoestic-products\",\n",
    "        body={\n",
    "            \"mappings\": {\n",
    "                \"properties\": {\n",
    "                    \"product_id\": {\"type\": \"keyword\"},\n",
    "                    \"product_name\": {\"type\": \"text\"},\n",
    "                    \"amount\": {\"type\": \"float\"},\n",
    "                    \"tags\": {\"type\": \"keyword\"},\n",
    "                }\n",
    "            }\n",
    "        },\n",
    "    )\n",
    "except exceptions.RequestError as e:\n",
    "    if e.error == \"resource_already_exists_exception\":\n",
    "        print(\"Index already exists.\")\n",
    "    else:\n",
    "        raise e"
   ],
   "metadata": {
    "id": "DG9Zi1LdZeUd"
   },
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    "# For data in docx\n",
    "try:\n",
    "    es_client.indices.create(\n",
    "        index=\"shoestic-products-descriptions\",\n",
    "        body={\n",
    "            \"mappings\": {\n",
    "                \"properties\": {\n",
    "                    \"title\": {\"type\": \"text\", \"analyzer\": \"english\"},\n",
    "                    \"super_body\": {\n",
    "                        \"type\": \"semantic_text\",\n",
    "                        \"inference_id\": \"onelake-inference-endpoint\",\n",
    "                    },\n",
    "                    \"body\": {\"type\": \"text\", \"copy_to\": \"super_body\"},\n",
    "                }\n",
    "            }\n",
    "        },\n",
    "    )\n",
    "except exceptions.RequestError as e:\n",
    "    if e.error == \"resource_already_exists_exception\":\n",
    "        print(\"Index already exists.\")\n",
    "    else:\n",
    "        raise e"
   ],
   "metadata": {
    "id": "G7k6YiQehzBf"
   },
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Indexing data"
   ],
   "metadata": {
    "id": "HOkYO6LtZkQC"
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "### Retrieving documents from OneLake"
   ],
   "metadata": {
    "id": "LDyqhYao9gpQ"
   }
  },
  {
   "cell_type": "code",
   "source": [
    "# Getting files from Lakehouse\n",
    "csv_file_client = get_file_by_name(\"products.csv\", directory_client)\n",
    "\n",
    "docx_files_clients = []\n",
    "\n",
    "for docx_file_name in docx_files:\n",
    "    docx_files_clients.append(get_file_by_name(docx_file_name, directory_client))"
   ],
   "metadata": {
    "id": "vTIsi_THo8ws"
   },
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "markdown",
   "source": [
    "### Extract content"
   ],
   "metadata": {
    "id": "wjPw7wxHZ_w0"
   }
  },
  {
   "cell_type": "code",
   "source": [
    "csv_content = get_csv_content(csv_file_client)\n",
    "reader = csv.DictReader(StringIO(csv_content))\n",
    "\n",
    "docx_contents = []\n",
    "\n",
    "for docx_file_client in docx_files_clients:\n",
    "    docx_contents.append(get_docx_content(docx_file_client))\n",
    "\n",
    "print(\"CSV FILE CONTENT: \", csv_content)\n",
    "print(\"DOCX FILE CONTENT: \", docx_contents)"
   ],
   "metadata": {
    "id": "WrvwaqZ4_t5U",
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "outputId": "27241c26-0725-494f-96d0-97383327a334"
   },
   "execution_count": null,
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "CSV FILE CONTENT:  product_id,product_name,amount,tags\n",
      "P-115,Classic Loafers,19.69,\"classic,formal\"\n",
      "P-114,Sport Sneakers,156.18,\"sport,casual\"\n",
      "P-109,Ankle Boots,87.13,\"boots,winter\"\n",
      "P-118,Casual Sandals,128.22,\"casual,summer\"\n",
      "P-112,Running Shoes,132.27,\"sport,running\"\n",
      "P-116,Leather Sandals,107.63,\"leather,casual\"\n",
      "P-105,Canvas Sneakers,75.56,\"casual,sneakers\"\n",
      "P-107,Winter Boots,49.57,\"winter,outdoor\"\n",
      "P-111,Formal Oxfords,177.70,\"formal,classic\"\n",
      "P-119,Beach Flip-Flops,50.64,\"beach,summer\"\n",
      "\n",
      "DOCX FILE CONTENT:  ['Crafted for an active lifestyle, this footwear offers excellent support and durability, featuring a non-slip sole that’s perfect for both workouts and casual use.', 'With a timeless design, this elegant choice pairs well with professional and social attire. Made from high-quality leather, it adds a touch of sophistication.', 'Ideal for warm, sunny days by the water, these lightweight essentials are water-resistant and come in bright colors, bringing a laid-back vibe to any outing in the sun.']\n"
     ]
    }
   ]
  },
  {
   "cell_type": "code",
   "source": [
    "# The tags must be an array\n",
    "rows = csv_content.splitlines()\n",
    "reader = csv.DictReader(rows)\n",
    "\n",
    "modified_rows = []\n",
    "\n",
    "for row in reader:\n",
    "    row[\"tags\"] = row[\"tags\"].replace('\"', \"\").split(\",\")\n",
    "    modified_rows.append(row)\n",
    "\n",
    "    print(row[\"tags\"])\n",
    "\n",
    "reader = modified_rows"
   ],
   "metadata": {
    "id": "vp7skddC_VIJ",
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "outputId": "2608ddcf-cf4a-4681-eb53-101e21dd6a51"
   },
   "execution_count": null,
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "['classic', 'formal']\n",
      "['sport', 'casual']\n",
      "['boots', 'winter']\n",
      "['casual', 'summer']\n",
      "['sport', 'running']\n",
      "['leather', 'casual']\n",
      "['casual', 'sneakers']\n",
      "['winter', 'outdoor']\n",
      "['formal', 'classic']\n",
      "['beach', 'summer']\n"
     ]
    }
   ]
  },
  {
   "cell_type": "markdown",
   "source": [
    "### Indexing data"
   ],
   "metadata": {
    "id": "qeWE60zy-MQs"
   }
  },
  {
   "cell_type": "code",
   "source": [
    "csv_actions = [{\"_index\": \"shoestic-products\", \"_source\": row} for row in reader]\n",
    "\n",
    "docx_actions = [\n",
    "    {\n",
    "        \"_index\": \"shoestic-products-descriptions\",\n",
    "        \"_source\": {\"title\": docx_file_name, \"body\": docx},\n",
    "    }\n",
    "    for docx_file_name, docx in zip(docx_files, docx_contents)\n",
    "]\n",
    "\n",
    "\n",
    "helpers.bulk(es_client, csv_actions)\n",
    "print(\"CSV data indexed successfully.\")\n",
    "helpers.bulk(es_client, docx_actions)\n",
    "print(\"DOCX data indexed successfully.\")"
   ],
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "WfIyoWoD-Rsv",
    "outputId": "25732e05-c048-4a2a-86dd-1561e8c21872"
   },
   "execution_count": null,
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "CSV data indexed successfully.\n",
      "DOCX data indexed successfully.\n"
     ]
    }
   ]
  },
  {
   "cell_type": "markdown",
   "source": [
    "### Retrieving data from Elasticsearch"
   ],
   "metadata": {
    "id": "HrPYKMxf-IV8"
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "Semantic search: let's search for the word 'summer' and see what results the search returns."
   ],
   "metadata": {
    "id": "r1Q0oyFLpgDN"
   }
  },
  {
   "cell_type": "code",
   "source": [
    "response = es_client.search(\n",
    "    index=\"shoestic-products-descriptions\",\n",
    "    body={\n",
    "        \"size\": 1,\n",
    "        \"_source\": {\"excludes\": [\"*embeddings\", \"*chunks\"]},\n",
    "        \"query\": {\"semantic\": {\"field\": \"super_body\", \"query\": \"summer\"}},\n",
    "    },\n",
    ")\n",
    "\n",
    "print(response[\"hits\"][\"hits\"][0][\"_source\"][\"body\"])"
   ],
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "Ij4Vqbq1o23q",
    "outputId": "60b940f5-fcaa-496c-c77a-309e3564cfe0"
   },
   "execution_count": null,
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Ideal for warm, sunny days by the water, these lightweight essentials are water-resistant and come in bright colors, bringing a laid-back vibe to any outing in the sun.\n"
     ]
    }
   ]
  },
  {
   "cell_type": "markdown",
   "source": [
    "Now let's search for the same word in the product tags using a full-text search. Both responses are related."
   ],
   "metadata": {
    "id": "wabN70Ixq9tA"
   }
  },
  {
   "cell_type": "code",
   "source": [
    "response = es_client.search(\n",
    "    index=\"shoestic-products\", body={\"size\": 1, \"query\": {\"term\": {\"tags\": \"summer\"}}}\n",
    ")\n",
    "\n",
    "print(response[\"hits\"][\"hits\"][0][\"_source\"])"
   ],
   "metadata": {
    "id": "wE8mg2IaZ5sg",
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "outputId": "f5aa9781-ace4-44e7-d6b8-8d7fe9c6ae70"
   },
   "execution_count": null,
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "{'product_id': 'P-118', 'product_name': 'Casual Sandals', 'amount': '128.22', 'tags': ['casual', 'summer']}\n"
     ]
    }
   ]
  },
  {
   "cell_type": "markdown",
   "source": [
    "## Cleanup\n",
    "\n",
    "Finally, we can delete the resources used to prevent them from consuming resources."
   ],
   "metadata": {
    "id": "UJrHLf3QaI2c"
   }
  },
  {
   "cell_type": "code",
   "source": [
    "# Cleanup - Delete Index\n",
    "es_client.indices.delete(index=\"shoestic-products\", ignore=[400, 404])\n",
    "es_client.indices.delete(index=\"shoestic-products-descriptions\", ignore=[400, 404])\n",
    "\n",
    "# Cleanup - Delete Embeddings Endpoint\n",
    "es_client.inference.delete_model(\n",
    "    inference_id=\"onelake-inference-endpoint\", ignore=[400, 404]\n",
    ")"
   ],
   "metadata": {
    "id": "LaCgcAEjaFmV",
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "outputId": "0b661e96-908d-4c55-b7a8-87244f39ab22"
   },
   "execution_count": null,
   "outputs": [
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "<ipython-input-16-a48b0e63148f>:2: DeprecationWarning: Passing transport options in the API method is deprecated. Use 'Elasticsearch.options()' instead.\n",
      "  es_client.indices.delete(index=\"shoestic-products\", ignore=[400, 404])\n",
      "<ipython-input-16-a48b0e63148f>:3: DeprecationWarning: Passing transport options in the API method is deprecated. Use 'Elasticsearch.options()' instead.\n",
      "  es_client.indices.delete(index=\"shoestic-products-descriptions\", ignore=[400, 404])\n",
      "<ipython-input-16-a48b0e63148f>:6: DeprecationWarning: Passing transport options in the API method is deprecated. Use 'Elasticsearch.options()' instead.\n",
      "  es_client.inference.delete_model(inference_id=\"onelake-inference-endpoint\", ignore=[400, 404])\n"
     ]
    },
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "ObjectApiResponse({'acknowledged': True, 'pipelines': []})"
      ]
     },
     "metadata": {},
     "execution_count": 16
    }
   ]
  }
 ]
}